'''
Author: 奔跑的乌龟
Date: 2020-12-05 13:25:53
LastEditTime: 2020-12-05 18:19:08
email: 435327238@qq.com
FilePath: \waibaoe:\爬虫12-4\Db.py
'''
import requests
from lxml import etree
import html
import time
import gethtml
import Db
        
def analysisHtml(url,booknumber):

    try:
        res = gethtml.getHtml(url)

        # 判断数据有效性
        if (res=="1"):
            print("error,return 1")
            # return "1"

        # 解析操作
        selector = etree.HTML(res.text.encode('ISO-8859-1').decode('UTF-8'))

        selector = etree.HTML(res.text.encode('ISO-8859-1').decode('UTF-8'))
        author = selector.xpath('//span/text()')[1]
        title = selector.xpath('//a/text()')[9]
        cover = selector.xpath('//meta/@content')[13]
        bind = selector.xpath('//meta/@content')[14] 

        # 组装简介
        synopois = '';
        i = 0;
        while (True):
            synopois += selector.xpath('//div[@class="intro"]/p/text()')[i]
            i += 1;
            if (i >= 4):
                break

        # 定义book数组存储数据
        bookinfo = [booknumber,author,title,bind,cover,synopois]
        sql = 'insert into bookinfo(booknumber,author,title,bind,cover,synopsis) values(%s,%s,%s,%s,%s,%s);'
        Db.insert(sql,bookinfo)
    except BaseException:
        return "1" 

def produceurl():
    
    # 查询数据库已存储的数据最大booknumber值，如果没有则从1开始爬取
    try:
        value = "SELECT * FROM bookinfo WHERE  booknumber=(SELECT MAX(booknumber) FROM bookinfo);"
        maxBookNumber = Db.query(value)
        maxBookNumber = maxBookNumber[0][1]
    except BaseException:
        maxBookNumber = 1

    stop = 120000
    while(True):
        
        url = 'http://www.aomolit.com/t/' + str(maxBookNumber)
        result = analysisHtml(url,maxBookNumber)
        maxBookNumber += 1
        if (result=="1"):
            print("该数据不存在，数据连接是："+url+"，将跳过该数据！")
            continue
        print("爬取成功url是："+url)
        if (maxBookNumber>=stop):
            break    
            
if __name__=="__main__":
    produceurl()